package net.anzhuo.common.utils;

import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

public class MyCrawler extends WebCrawler {

    /**
     * This method receives two parameters. The first parameter is the page
     * in which we have discovered this new url and the second parameter is
     * the new url. You should implement this function to specify whether
     * the given url should be crawled or not (based on your crawling logic).
     * In this example, we are instructing the crawler to ignore urls that
     * have css, js, git, ... extensions and to only accept urls that start
     * with "http://www.ics.uci.edu/". In this case, we didn't need the
     * referringPage parameter to make the decision.
     */
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
         String href = url.getURL().toLowerCase();
         return true;
     }
 
     /**
      * This function is called when a page is fetched and ready
      * to be processed by your program.
      */
     @Override
     public void visit(Page page) {
         String url = page.getWebURL().getURL();
         System.out.println("URL: " + url);
 
         if (page.getParseData() instanceof HtmlParseData) {
             HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
             String result="";
     		Document doc=Jsoup.parse(page.toString());
     		Element elem = doc.getElementById("searchlist");
     		if(elem!=null)
     		{
     		 Elements	TableElem=elem.getElementsByClass("m_srchList");
     		 if(TableElem!=null&&TableElem.size()>0)
     		 {
     			 Element table=TableElem.get(0);
     			 Elements trlist=table.getElementsByTag("tr");
     			 if(trlist!=null&&trlist.size()>0)
     			 {
     				Element tr1= trlist.get(1);
     				
     				Elements tds=tr1.getElementsByTag("td");
     				
     				if(tds!=null&&tds.size()>0)
     				{
     					Element td2=tds.get(1);
     					
     					Elements as=td2.getElementsByTag("a");
     					
     					if(as!=null&&as.size()>0)
     					{
     					  Element a=as.get(0);	  
     					  result=a.attr("href");
     					  result=result.replace("/firm_", "");
     					  result=result.replace(".html","");
     					}
     				}
     				
     			 }
     		 }
     		}
         }
    }
}
